{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"CH03.1 As one of Autoencoding Language Models.ipynb","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"5661f9afceac41cd81911ec61f840d1c":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_0fe97a5a63614b35b1a58cb8d9765242","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_6295bed1bbfc43ff8bd374152775b094","IPY_MODEL_91e3626b49814c7e85fc351d6cea403c"]}},"0fe97a5a63614b35b1a58cb8d9765242":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6295bed1bbfc43ff8bd374152775b094":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_cd7b13f0ba0f44a88b1d085fb5acf789","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":570,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":570,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_24e111b1fc5746a8a69f862a0dc3bc7d"}},"91e3626b49814c7e85fc351d6cea403c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_e345fb951df24e0d848a051a485b4b65","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 570/570 [00:01&lt;00:00, 327B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_86b7927b6dc44fb28cf3b006163c39cf"}},"cd7b13f0ba0f44a88b1d085fb5acf789":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"24e111b1fc5746a8a69f862a0dc3bc7d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e345fb951df24e0d848a051a485b4b65":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"86b7927b6dc44fb28cf3b006163c39cf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"133035cbbade4827bc6764218387a54b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_e4efd2d54399463db446591d18cbe6fd","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_c1e03b6d025a4dba9e37eb35c48d93ec","IPY_MODEL_62399066111345ec827e1912ec1d97ba"]}},"e4efd2d54399463db446591d18cbe6fd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c1e03b6d025a4dba9e37eb35c48d93ec":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_73ca0820cf8a482bb3cb4bd658e7a5d3","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":536063208,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":536063208,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2989c05594ea4054b311517ce5339ed2"}},"62399066111345ec827e1912ec1d97ba":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ef71247bb53a4c709d59e63ab7d60c33","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 536M/536M [00:30&lt;00:00, 17.3MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_62b92d97fdcb49b1ade6d076d0b203eb"}},"73ca0820cf8a482bb3cb4bd658e7a5d3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"2989c05594ea4054b311517ce5339ed2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ef71247bb53a4c709d59e63ab7d60c33":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"62b92d97fdcb49b1ade6d076d0b203eb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"94a9c2fcf79f4ac0a8f2d712411f7d2b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_d8e8c28122024aadb70ff1aab5530eeb","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_7a9083129f274149a176a0e31eb48411","IPY_MODEL_dac236c954ef4b0f99067b8bbeba6283"]}},"d8e8c28122024aadb70ff1aab5530eeb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7a9083129f274149a176a0e31eb48411":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_96ad3e5ee6bc4398a0c6e8ea948682a4","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":231508,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":231508,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3f2e7ff1d0ad466196075aad10b8b84a"}},"dac236c954ef4b0f99067b8bbeba6283":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9fb9172e15b3405fab9a7db14d0c560d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 232k/232k [00:00&lt;00:00, 798kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_48f55d9d4e3046d5abe21fef5b2a377c"}},"96ad3e5ee6bc4398a0c6e8ea948682a4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"3f2e7ff1d0ad466196075aad10b8b84a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9fb9172e15b3405fab9a7db14d0c560d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"48f55d9d4e3046d5abe21fef5b2a377c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7d102900f8884e44aa4953d89ad79465":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_6946d2105b2e4fd1bfab2b10f1256053","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_0b351be38ce9450eb4f4a857132b0a79","IPY_MODEL_c41a59274c78422fa1c846a1711b2439"]}},"6946d2105b2e4fd1bfab2b10f1256053":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0b351be38ce9450eb4f4a857132b0a79":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_477900896cec431f98ed705f5fbbf78b","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":466062,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":466062,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6cead6beda334e7a8db3f27ad5845705"}},"c41a59274c78422fa1c846a1711b2439":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d26b4d4dbcd243cd8933f81fffad0b4d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 466k/466k [00:00&lt;00:00, 2.94MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a0d2c4f1560a4a6bab0be1285c932283"}},"477900896cec431f98ed705f5fbbf78b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6cead6beda334e7a8db3f27ad5845705":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d26b4d4dbcd243cd8933f81fffad0b4d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a0d2c4f1560a4a6bab0be1285c932283":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9dac89e1bf484a1fafaa1b5235846d93":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_2290a421c99944a19da4fa1888204a0a","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3fe4f71880a5497e950d15a381614651","IPY_MODEL_7fc1dee184a54de987dbba3af55c597e"]}},"2290a421c99944a19da4fa1888204a0a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3fe4f71880a5497e950d15a381614651":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4e9a142be38247088fe4f3a7d6ecd6f3","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":28,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":28,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_127e05f39b2d4b0f84dd7ca57d71cc98"}},"7fc1dee184a54de987dbba3af55c597e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_73de1777b2f5478891fc9b8dc46c26a0","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 28.0/28.0 [00:08&lt;00:00, 3.33B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_476bdde5624144919c7abb27938c7221"}},"4e9a142be38247088fe4f3a7d6ecd6f3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"127e05f39b2d4b0f84dd7ca57d71cc98":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"73de1777b2f5478891fc9b8dc46c26a0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"476bdde5624144919c7abb27938c7221":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"v61w8HXuZrLH"},"source":["# BERT: As one of Autoencoding Language Models "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hXoOOoBAZqHq","executionInfo":{"status":"ok","timestamp":1625504058946,"user_tz":-180,"elapsed":363,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"cee84e9b-a02f-4908-a1ac-90c219372c54"},"source":["import os\n","from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t3bcOfnNWJhv","executionInfo":{"status":"ok","timestamp":1625504063059,"user_tz":-180,"elapsed":2921,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"42612c4a-b889-4cc1-e79c-1843478b88d0"},"source":["!pip install transformers"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.8.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: huggingface-hub==0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.12)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n","Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.5.0)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MDuHvNUnbRXu","executionInfo":{"status":"ok","timestamp":1625504065244,"user_tz":-180,"elapsed":2190,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"2a545f55-76da-494a-873d-b0f7419a73ad"},"source":["!pip install tokenizers"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: tokenizers in /usr/local/lib/python3.7/dist-packages (0.10.3)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"YR3-I4bFUVdm","executionInfo":{"status":"ok","timestamp":1625504065514,"user_tz":-180,"elapsed":4,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["os.chdir(\"drive/My Drive/data/\")"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MVadDu1T4NZF","executionInfo":{"status":"ok","timestamp":1625504065933,"user_tz":-180,"elapsed":4,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"888a80cf-7428-43ec-b5d6-0158a961916d"},"source":["os.listdir()"],"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['IMDB Dataset.csv']"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"hGJiQ2Sx4mm7","executionInfo":{"status":"ok","timestamp":1625504066616,"user_tz":-180,"elapsed":8,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":[""],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"sJbh3P-e3g0S","executionInfo":{"status":"ok","timestamp":1625504070634,"user_tz":-180,"elapsed":2870,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["import pandas as pd\n","imdb_df = pd.read_csv(\"IMDB Dataset.csv\")\n","reviews = imdb_df.review.to_string(index=None) \n","with open(\"corpus.txt\", \"w\") as f: \n","    f.writelines(reviews) "],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"JQRzoQKS4SA2","executionInfo":{"status":"ok","timestamp":1625504074342,"user_tz":-180,"elapsed":1783,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["from tokenizers import BertWordPieceTokenizer\n","bert_wordpiece_tokenizer = BertWordPieceTokenizer() \n","bert_wordpiece_tokenizer.train(\"corpus.txt\") "],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"datICiehV7kv","executionInfo":{"status":"ok","timestamp":1625504075440,"user_tz":-180,"elapsed":13,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"d4dce1dc-f6b9-4002-faae-c978f9c5ef62"},"source":["bert_wordpiece_tokenizer.get_vocab()"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'princess': 5136,\n"," 'stupidly': 15897,\n"," 'express': 3012,\n"," '##ilyn': 12285,\n"," 'circl': 16886,\n"," 'finds': 3069,\n"," 'heading': 16175,\n"," '##eday': 11710,\n"," 'fever': 4354,\n"," 'losing': 6946,\n"," 'december': 6523,\n"," '##ety': 8852,\n"," 'unemployed': 17922,\n"," '##omat': 11749,\n"," 'bits': 6375,\n"," 'apartment': 9412,\n"," 'mercenary': 17797,\n"," '##astimil': 14917,\n"," 'conclude': 17813,\n"," 'greystoke': 18010,\n"," 'enthusi': 4863,\n"," 'sm': 811,\n"," 'chucky': 5365,\n"," 'daniel': 3963,\n"," 'sweeney': 7093,\n"," 'broadway': 3342,\n"," 'eleph': 6413,\n"," '1999': 3712,\n"," 'coll': 3232,\n"," '##abe': 15117,\n"," 'unh': 8037,\n"," 'tol': 5817,\n"," 'dizz': 14919,\n"," 'aardman': 11220,\n"," '##ilation': 11703,\n"," 'theme': 3333,\n"," 'shi': 7963,\n"," 'thoug': 4260,\n"," 'godzilla': 9140,\n"," 'oft': 8808,\n"," 'enthralling': 13716,\n"," 'revival': 16467,\n"," '81': 13858,\n"," 'watch': 225,\n"," 'prepared': 7692,\n"," 'hamlet': 3984,\n"," 'iden': 16082,\n"," 'trashed': 12800,\n"," 'american': 947,\n"," 'indiana': 16668,\n"," 'malefique': 13233,\n"," 'cujo': 13025,\n"," 'produced': 2212,\n"," 'gein': 9142,\n"," 'helena': 16154,\n"," 'plankton': 18200,\n"," 'sour': 7846,\n"," 'extraordinarily': 16850,\n"," 'exer': 4507,\n"," 'historica': 16301,\n"," 'smoking': 7449,\n"," '##antic': 8001,\n"," 'longoria': 10422,\n"," 'feeling': 2382,\n"," 'immense': 5103,\n"," 'motivation': 17674,\n"," 'burtyns': 16032,\n"," '##phony': 15352,\n"," 'tets': 14128,\n"," 'kieslowski': 9688,\n"," 'isa': 14456,\n"," 'foppish': 18065,\n"," 'phoolan': 12412,\n"," 'tribe': 6883,\n"," 'tan': 14119,\n"," 'iyer': 13985,\n"," 'burrough': 9238,\n"," 'heston': 10277,\n"," 'amiable': 14861,\n"," 'pe': 373,\n"," 'pinocchio': 17711,\n"," 'richa': 15905,\n"," 'hen': 2484,\n"," 'seduction': 17457,\n"," 'broadcast': 4105,\n"," 'andersson': 13446,\n"," 'concent': 8925,\n"," 'amidst': 11941,\n"," '$': 8,\n"," '##opher': 2749,\n"," 'spi': 10135,\n"," '##ial': 841,\n"," 'iow': 11420,\n"," 'barbet': 6932,\n"," 'necess': 4512,\n"," 'had': 315,\n"," 'yaw': 11542,\n"," 'wannabe': 9627,\n"," 'christm': 15704,\n"," '##arians': 8812,\n"," 'defined': 10498,\n"," 'drive': 3973,\n"," 'hoopla': 13719,\n"," '##ari': 5489,\n"," 'lane': 4656,\n"," '##gingly': 14301,\n"," 'abomination': 8559,\n"," '##ley': 1151,\n"," '##ely': 466,\n"," 'story': 366,\n"," 'montana': 13118,\n"," 'lundgren': 6570,\n"," 'unti': 16142,\n"," 'outlaw': 10193,\n"," 'fistful': 17696,\n"," 'later': 2597,\n"," 'succeeded': 17004,\n"," '##bg': 8739,\n"," '##25': 14382,\n"," 'misadventures': 18241,\n"," 'meandering': 11960,\n"," 'mout': 14442,\n"," 'paradise': 11199,\n"," 'aust': 1266,\n"," 'ko': 5453,\n"," 'pinn': 17082,\n"," 'zandalee': 18140,\n"," 'incest': 13660,\n"," 'strays': 15427,\n"," 'amazin': 12386,\n"," 'mash': 8687,\n"," '1964': 12765,\n"," 'blowing': 15175,\n"," '##peg': 15042,\n"," 'continuation': 16606,\n"," 'evie': 13915,\n"," 'insulted': 8400,\n"," '##ified': 4205,\n"," 'carre': 15322,\n"," 'sakura': 14744,\n"," '##ube': 11613,\n"," 'comb': 2208,\n"," 'helps': 6943,\n"," 'freebird': 10759,\n"," 'bad': 326,\n"," '##semble': 3738,\n"," 'aspect': 2236,\n"," 'rockne': 12609,\n"," 'exists': 7977,\n"," 'appeared': 4418,\n"," '##ra': 262,\n"," 'rus': 2898,\n"," 'rh': 4905,\n"," 'ref': 1745,\n"," 'grandad': 16517,\n"," '##rama': 3041,\n"," 'unfortunate': 4090,\n"," 'cott': 13892,\n"," 'mcadams': 17984,\n"," 'tart': 8714,\n"," 'flix': 12178,\n"," 'amore': 14858,\n"," 'prophetic': 16400,\n"," 'yea': 3009,\n"," 'downhill': 15607,\n"," 'turgid': 7562,\n"," 'paced': 2514,\n"," 'uwe': 4480,\n"," 'stink': 2339,\n"," '##ulter': 11879,\n"," 'documen': 17584,\n"," 'chuck': 2462,\n"," 'divorce': 8568,\n"," '##inqu': 14413,\n"," 'hassan': 17908,\n"," 'prop': 2119,\n"," 'prej': 7334,\n"," 'rumours': 13720,\n"," 'neo': 8900,\n"," 'terri': 6317,\n"," 'engag': 2725,\n"," 'archie': 13128,\n"," 'currie': 16669,\n"," 'corman': 4782,\n"," 'devils': 16736,\n"," 'beastmaster': 17281,\n"," 'wilkinson': 18274,\n"," '11th': 9305,\n"," 'filmma': 11752,\n"," 'franco': 3251,\n"," 'ik': 13978,\n"," 'dimen': 17175,\n"," 'insults': 16845,\n"," '##ender': 4735,\n"," 'purposes': 10982,\n"," 'features': 2149,\n"," 'manipulative': 9575,\n"," '##fevres': 15127,\n"," '##ond': 385,\n"," 'pure': 1876,\n"," '##ads': 6705,\n"," 'loaned': 17903,\n"," 'comprised': 14733,\n"," 'fantasy': 2136,\n"," 'burning': 8246,\n"," 'premiered': 5115,\n"," 'tenacious': 9193,\n"," '##oki': 14257,\n"," 'infinite': 16211,\n"," '##yad': 8777,\n"," 'cree': 11367,\n"," 'rohm': 17099,\n"," '##icance': 12378,\n"," 'xavier': 11538,\n"," 'fantasti': 11039,\n"," 'couldnt': 10328,\n"," '##olo': 10131,\n"," 'happe': 13448,\n"," 'enigma': 17738,\n"," 'meticulously': 17854,\n"," 'myth': 6729,\n"," 'rockwell': 15945,\n"," '1975': 4575,\n"," 'says': 1861,\n"," 'spor': 11924,\n"," 'playboy': 12324,\n"," 'jersey': 16393,\n"," '##lein': 8826,\n"," 'muddled': 9709,\n"," 'in': 163,\n"," '##si': 9906,\n"," '68': 11343,\n"," 'spencer': 14845,\n"," 'morni': 15798,\n"," 'nest': 11975,\n"," 'took': 1364,\n"," 'blaxploit': 17242,\n"," 'isab': 5815,\n"," 'testoster': 17052,\n"," 'caddy': 6530,\n"," 'fig': 2193,\n"," 'abbott': 6129,\n"," 'broug': 14647,\n"," 'outline': 8917,\n"," 'clueless': 13240,\n"," 'carefully': 10614,\n"," '##uel': 2113,\n"," 'classy': 9042,\n"," 'wicked': 7032,\n"," 'hardware': 9045,\n"," '##ast': 295,\n"," 'millions': 6019,\n"," 'nord': 9116,\n"," 'clon': 10251,\n"," 'predictable': 2012,\n"," 'nintendo': 13589,\n"," 'specia': 15532,\n"," 'bea': 7938,\n"," 'engaged': 10823,\n"," '##etr': 14665,\n"," 'ordinary': 5111,\n"," 'reserv': 12676,\n"," 'ranma': 16643,\n"," 'wizards': 17506,\n"," '##aro': 9981,\n"," 'randy': 9378,\n"," 'theatric': 15733,\n"," '##math': 9901,\n"," 'kink': 11434,\n"," 'rag': 7837,\n"," '##propriate': 12950,\n"," 'janet': 13094,\n"," '##asph': 11701,\n"," 'skilled': 13462,\n"," 'deathly': 16041,\n"," 'foc': 2866,\n"," 'nightbre': 6804,\n"," 'nolan': 4636,\n"," 'accept': 4370,\n"," 'decept': 7310,\n"," '##alondo': 14520,\n"," '##chily': 14600,\n"," 'core': 3267,\n"," '##go': 2335,\n"," 'h': 50,\n"," '##ify': 3320,\n"," 'scared': 4817,\n"," 'owns': 12679,\n"," 'dere': 4882,\n"," 'barricade': 18243,\n"," '##cast': 2786,\n"," 'ya': 4154,\n"," 'suckered': 10608,\n"," 'ª': 78,\n"," 'bears': 6985,\n"," 'psycho': 5356,\n"," 'bly': 13873,\n"," '##gut': 7879,\n"," 'ment': 4143,\n"," '##fil': 9942,\n"," 'eat': 4303,\n"," 'meteorite': 17301,\n"," 'evangel': 13699,\n"," '##cifully': 15279,\n"," 'brothe': 16350,\n"," 'bank': 5746,\n"," '##ortion': 8041,\n"," 'ich': 7813,\n"," 'schrader': 13617,\n"," 'glimpse': 8545,\n"," 'cris': 6303,\n"," 'darwin': 12802,\n"," 'heat': 7358,\n"," '##leng': 8766,\n"," 'visiteurs': 10591,\n"," 'allen': 2621,\n"," 'actor': 757,\n"," 'remo': 15131,\n"," 'bubbling': 17643,\n"," 'recre': 7384,\n"," '##end': 456,\n"," 'relish': 12498,\n"," 'traw': 15384,\n"," 'unspe': 8933,\n"," 'loosel': 12854,\n"," 'ape': 7109,\n"," 'sprawling': 8614,\n"," '##inski': 9282,\n"," 'garde': 13123,\n"," 'electrifying': 13323,\n"," 'day': 991,\n"," 'lumet': 9529,\n"," 'lovable': 13111,\n"," 'listener': 8427,\n"," 'topic': 5971,\n"," '##rid': 3766,\n"," 'stunn': 12709,\n"," 'kas': 9791,\n"," 'sometimes': 1600,\n"," 'dece': 4183,\n"," 'woo': 7182,\n"," 'engage': 15821,\n"," '##just': 7211,\n"," 'rated': 1712,\n"," 'deaf': 9516,\n"," 'ivey': 13984,\n"," '##ipse': 12340,\n"," 'os': 7160,\n"," '##avano': 15522,\n"," 'mcdo': 16293,\n"," 'collabor': 6989,\n"," 'wis': 9864,\n"," 'maximum': 17815,\n"," 'larg': 16093,\n"," 'remind': 2180,\n"," 'sacr': 17087,\n"," 'hybrid': 6419,\n"," 'vac': 3560,\n"," '1990': 3005,\n"," 'body': 3874,\n"," 'allegedly': 9624,\n"," 'kib': 11436,\n"," 'primitive': 16941,\n"," 'peaches': 17926,\n"," 'effec': 13627,\n"," 'romp': 4209,\n"," 'guardian': 17417,\n"," '##onda': 15086,\n"," 'dep': 1278,\n"," 'yet': 995,\n"," 'hints': 13973,\n"," 'inc': 795,\n"," 'fault': 5535,\n"," 'reconcile': 18041,\n"," 'lucy': 6842,\n"," 'pyramid': 17226,\n"," '6th': 7786,\n"," 'boiler': 13367,\n"," 'field': 6596,\n"," 'longing': 12299,\n"," 'rema': 8068,\n"," 'contemporary': 9601,\n"," 'altern': 4048,\n"," '##aming': 2744,\n"," '##00': 564,\n"," 'forest': 6248,\n"," '##ghter': 8861,\n"," 'falk': 6049,\n"," 'utterly': 2596,\n"," 'pict': 1085,\n"," 'wid': 2783,\n"," '##agraph': 8988,\n"," 'harlem': 10509,\n"," 'butter': 7995,\n"," 'jewel': 10923,\n"," '##ami': 7959,\n"," '##fore': 9944,\n"," 'uproarious': 17929,\n"," 'intell': 2310,\n"," 'fess': 13935,\n"," '##kes': 1597,\n"," '##o': 96,\n"," 'overlooked': 5104,\n"," 'kathy': 7017,\n"," 'roberts': 6395,\n"," 'teenager': 2590,\n"," '##selor': 16582,\n"," '1916': 16969,\n"," 'slackers': 17637,\n"," 'mamet': 7025,\n"," 'everybody': 2864,\n"," 'deceptively': 13495,\n"," 'macbeth': 8599,\n"," 'extraord': 3424,\n"," 'seriou': 17124,\n"," 'ishimoto': 17149,\n"," '##uall': 8765,\n"," '##zac': 14354,\n"," 'valid': 16070,\n"," 'haz': 7254,\n"," 'historically': 6928,\n"," 'wire': 7856,\n"," 'positives': 10735,\n"," 'mantis': 11463,\n"," 'caan': 15929,\n"," 'sequenc': 15745,\n"," 'safely': 9536,\n"," '##ued': 7196,\n"," 'jorg': 8671,\n"," 'involved': 3735,\n"," 'facil': 15019,\n"," 'prob': 678,\n"," 'encouraged': 11332,\n"," 'nunez': 18102,\n"," 'repetitive': 7757,\n"," 'environment': 6569,\n"," '26': 6136,\n"," 'points': 3157,\n"," 'place': 1843,\n"," 'stars': 909,\n"," 'myster': 3374,\n"," 'essay': 6951,\n"," '##ice': 539,\n"," 'webs': 3921,\n"," '##vement': 4944,\n"," 'bought': 1041,\n"," 'cbc': 11368,\n"," 'packs': 8347,\n"," 'cooley': 8413,\n"," 'dune': 13910,\n"," 'triad': 9214,\n"," 'performa': 7482,\n"," '##ila': 6684,\n"," 'godd': 12453,\n"," 'tomatoes': 9681,\n"," 'roc': 15145,\n"," 'sile': 9839,\n"," 'breed': 8240,\n"," 'uninvolving': 10964,\n"," '##burne': 16933,\n"," '##gan': 1680,\n"," 'quite': 649,\n"," '##acing': 15434,\n"," 'wesley': 5680,\n"," 'cock': 6324,\n"," 'manner': 7315,\n"," '##using': 2052,\n"," 'darling': 9317,\n"," '##guay': 12581,\n"," '##ious': 410,\n"," 'manna': 7314,\n"," 'cult': 1709,\n"," 'stargate': 8997,\n"," '##va': 4029,\n"," 'shows': 1095,\n"," 'lonesome': 7596,\n"," '##sman': 9909,\n"," 'bloch': 10316,\n"," '##gee': 14892,\n"," 'langella': 17852,\n"," '##ject': 1314,\n"," '##z': 109,\n"," 'repres': 3509,\n"," 'puzzle': 8549,\n"," '##enster': 14430,\n"," 'klap': 17312,\n"," '##iday': 2000,\n"," 'claiming': 10849,\n"," 'comfortable': 17611,\n"," '##don': 4682,\n"," 'stin': 14587,\n"," 'missile': 15888,\n"," 'danner': 15839,\n"," 'burn': 3873,\n"," 'iz': 9785,\n"," 'enab': 14875,\n"," 'lighthearted': 16073,\n"," 'centur': 10702,\n"," 'pract': 4177,\n"," 'knight': 4517,\n"," 'vanc': 7553,\n"," 'effective': 5652,\n"," '##see': 11847,\n"," '##and': 322,\n"," '##hey': 15892,\n"," 'loaded': 10117,\n"," 'trinity': 14970,\n"," 'adject': 10244,\n"," 'outra': 4716,\n"," 'finally': 1247,\n"," 'jungle': 7030,\n"," 'deba': 16212,\n"," '##yr': 9928,\n"," 'verb': 16013,\n"," 'glorious': 8175,\n"," 'woodward': 16139,\n"," 'doll': 2657,\n"," 'swanson': 11255,\n"," 'ust': 7852,\n"," 'trekkies': 16637,\n"," 'curve': 15844,\n"," 'absolute': 2007,\n"," '##reed': 14499,\n"," '##raiser': 7301,\n"," 'ollie': 18103,\n"," 'calling': 4566,\n"," '##estrian': 9018,\n"," '##irth': 14632,\n"," 'supply': 15734,\n"," 'shod': 10066,\n"," 'league': 3936,\n"," 'told': 1763,\n"," 'nimoy': 17855,\n"," 'cure': 5143,\n"," 'brilliant': 974,\n"," '##body': 1361,\n"," 'drom': 15345,\n"," 'geisha': 9145,\n"," '##girls': 9585,\n"," 'sy': 1565,\n"," 'morris': 6861,\n"," 'eas': 1083,\n"," 'crude': 6304,\n"," 'lank': 14027,\n"," 'hype': 3169,\n"," '##ican': 812,\n"," 'occasional': 16769,\n"," 'ealing': 6588,\n"," 'nich': 3886,\n"," 'tv': 515,\n"," 'fragile': 17468,\n"," 'ille': 6521,\n"," 'blown': 6302,\n"," 'quent': 5544,\n"," 'stunner': 16098,\n"," 'dictator': 13522,\n"," 'robots': 16168,\n"," 'sensitive': 5672,\n"," 'lynchian': 16770,\n"," 'dow': 5749,\n"," 'riget': 13337,\n"," '17th': 12996,\n"," 'paranormal': 9687,\n"," 'whoopi': 7072,\n"," 'shepitko': 11133,\n"," 'cain': 11364,\n"," 'intensity': 18208,\n"," 'timothy': 6756,\n"," 'doesnt': 12267,\n"," 'graduate': 13426,\n"," 'planning': 6427,\n"," 'production': 1097,\n"," '##ach': 801,\n"," '##e': 91,\n"," 'tak': 1878,\n"," 'kinda': 3071,\n"," 'him': 2043,\n"," 'especia': 8274,\n"," 'program': 2289,\n"," 'dots': 13908,\n"," 'extravagan': 16452,\n"," 'detective': 3644,\n"," 'grand': 2414,\n"," 'coul': 5012,\n"," 'decadent': 17951,\n"," 'rex': 6698,\n"," 'dock': 14869,\n"," 'leon': 3138,\n"," 'elements': 3839,\n"," 'century': 3622,\n"," 'instruc': 17884,\n"," '##ors': 1277,\n"," 'slight': 2241,\n"," 'brooke': 10050,\n"," 'damn': 3727,\n"," 'cosm': 8449,\n"," 'snore': 10583,\n"," 'lab': 3275,\n"," 'conce': 3475,\n"," 'hmmmm': 8434,\n"," 'unites': 15034,\n"," '##itism': 14515,\n"," 'thea': 7216,\n"," 'arqu': 12142,\n"," 'hyde': 8260,\n"," 'predecessor': 13736,\n"," '##bezz': 15415,\n"," '##heads': 9312,\n"," 'thrilled': 9398,\n"," '##rall': 9884,\n"," 'evid': 5022,\n"," '##stru': 4803,\n"," 'curly': 15843,\n"," 'montgo': 13119,\n"," 'ends': 3706,\n"," 'juliet': 6451,\n"," 'coming': 1731,\n"," 'recei': 16339,\n"," 'symb': 5395,\n"," 'tripe': 9215,\n"," 'lef': 10300,\n"," '##ished': 1294,\n"," '##cula': 4318,\n"," 'wol': 7854,\n"," 'deserves': 2355,\n"," 'rory': 11491,\n"," 'deceptive': 13494,\n"," 'gathered': 11029,\n"," 'es': 4302,\n"," 'unbelievable': 3087,\n"," 'warlock': 15306,\n"," 'histori': 16300,\n"," 'denzel': 12749,\n"," 'leo': 15136,\n"," '##oooo': 2742,\n"," 'quaid': 9591,\n"," 'unhappy': 9668,\n"," '##il': 147,\n"," 'react': 4699,\n"," 'underst': 3149,\n"," 'stark': 8993,\n"," 'hagg': 6716,\n"," 'barrel': 10471,\n"," 'taken': 2598,\n"," '##ind': 371,\n"," 'routine': 3997,\n"," '##etically': 14667,\n"," 'lasts': 15297,\n"," 'a': 43,\n"," 'correct': 5706,\n"," '##vaag': 16991,\n"," 'val': 1507,\n"," 'city': 1838,\n"," '##ures': 1118,\n"," 'tyler': 7447,\n"," 'suddenly': 11072,\n"," 'cannibalistic': 13425,\n"," 'shocked': 2881,\n"," 'exc': 1204,\n"," 'elisha': 12452,\n"," 'tramp': 8921,\n"," 'docudrama': 10853,\n"," '46': 11341,\n"," 'tugg': 14132,\n"," '##ß': 128,\n"," 'greatest': 1200,\n"," 'approached': 8395,\n"," '##wald': 14345,\n"," 'describing': 7576,\n"," 'horrendous': 3941,\n"," 'bou': 13872,\n"," 'lexington': 17932,\n"," 'former': 3125,\n"," 'axiom': 17377,\n"," 'gradu': 6500,\n"," 'ø': 86,\n"," 'rowan': 9546,\n"," 'object': 8160,\n"," 'awa': 4060,\n"," 'shorts': 3701,\n"," 'enjoyab': 12169,\n"," 'garbag': 12909,\n"," 'powerhouse': 16054,\n"," 'wishing': 8254,\n"," '##alta': 14519,\n"," '##enter': 14616,\n"," '##bi': 4025,\n"," '##rem': 1011,\n"," 'alas': 6740,\n"," 'dial': 1966,\n"," 'came': 723,\n"," 'lesley': 12884,\n"," 'merciless': 18271,\n"," 'cigar': 13740,\n"," 'lazy': 5769,\n"," 'rack': 5464,\n"," 'africa': 5679,\n"," 'projection': 11242,\n"," 'distrib': 6892,\n"," 'colman': 7438,\n"," 'helicop': 16155,\n"," 'lampoon': 4603,\n"," 'glowing': 8174,\n"," '##book': 9875,\n"," 'plight': 10182,\n"," 'eig': 13919,\n"," '1960': 6005,\n"," '##oblins': 9332,\n"," '##anted': 10158,\n"," 'lov': 3582,\n"," 'trif': 12613,\n"," 'strug': 15429,\n"," 'bulgaria': 18017,\n"," '##to': 2230,\n"," 'amount': 4179,\n"," '##field': 2843,\n"," 'naked': 4661,\n"," 'seriousl': 15803,\n"," 'explo': 6876,\n"," 'uninte': 13112,\n"," 'particular': 2551,\n"," 'descr': 12474,\n"," 'pbs': 5171,\n"," 'fred': 1906,\n"," 'hmm': 3878,\n"," '##borough': 9390,\n"," 'fabulous': 3089,\n"," 'gretchen': 17909,\n"," 'recomme': 8253,\n"," 'rajkumar': 13305,\n"," 'happen': 1344,\n"," 'claim': 2933,\n"," 'nevada': 16786,\n"," 'genius': 2380,\n"," 'take': 1021,\n"," 'slog': 15314,\n"," 'sar': 2361,\n"," 'baseball': 4271,\n"," 'jung': 4652,\n"," 'capital': 5347,\n"," '##iance': 6313,\n"," '##felt': 11646,\n"," '##enet': 14427,\n"," 'amicus': 10146,\n"," 'mannered': 13496,\n"," 'governmen': 13239,\n"," 'interwo': 12509,\n"," 'swit': 9177,\n"," 'separates': 13193,\n"," 'alleged': 7033,\n"," '##aude': 4714,\n"," 'stranded': 9050,\n"," 'beatty': 10799,\n"," 'incons': 15544,\n"," 'biblical': 9606,\n"," 'hoping': 2545,\n"," 'work': 698,\n"," 'riotously': 17492,\n"," '##ga': 3116,\n"," 'dumped': 17382,\n"," 'turning': 5067,\n"," 'caper': 6438,\n"," 'sou': 5784,\n"," 'histor': 1912,\n"," 'columbia': 6949,\n"," 'humble': 12486,\n"," 'st': 181,\n"," 'recall': 4366,\n"," 'types': 5620,\n"," 'geish': 15690,\n"," '##adel': 5832,\n"," 'rabbit': 8534,\n"," 'angeles': 3992,\n"," 'dismal': 12102,\n"," 'fligh': 15227,\n"," 'decide': 3707,\n"," 'house': 1047,\n"," '##vey': 4037,\n"," 'productions': 5615,\n"," 'feature': 1550,\n"," 'posters': 5974,\n"," '##ement': 2619,\n"," 'monke': 5021,\n"," 'arguably': 3863,\n"," 'ext': 900,\n"," 'bernsen': 11330,\n"," '##abul': 15119,\n"," 'resisted': 17661,\n"," '##ir': 192,\n"," 'rules': 4769,\n"," 'wolfe': 13531,\n"," 'violated': 16375,\n"," 'brie': 14641,\n"," 'haun': 2920,\n"," 'drool': 12249,\n"," 'sequence': 5961,\n"," 'nelson': 8554,\n"," 'voy': 4153,\n"," '##acao': 15436,\n"," 'concorde': 10929,\n"," 'winc': 10616,\n"," 'lucas': 7633,\n"," 'c': 45,\n"," 'nominated': 5114,\n"," 'acceptable': 13216,\n"," 'freshman': 8409,\n"," 'prett': 7404,\n"," '1928': 8372,\n"," '##ingers': 4938,\n"," 'reversal': 11231,\n"," '##elle': 2408,\n"," 'cloying': 12075,\n"," 'hope': 1752,\n"," 'off': 481,\n"," 'balls': 13075,\n"," 'thelma': 14409,\n"," 'purgatory': 16039,\n"," 'cheap': 1561,\n"," 'nons': 3753,\n"," 'kaddiddle': 18090,\n"," 'arizona': 11219,\n"," 'stress': 11789,\n"," 'browsing': 7073,\n"," '##usa': 12140,\n"," 'level': 2756,\n"," 'synchron': 16541,\n"," 'hepburn': 11320,\n"," 'few': 599,\n"," 'directors': 2377,\n"," '##ities': 2532,\n"," 'spoken': 8615,\n"," 'var': 1727,\n"," 'jumb': 11426,\n"," 'king': 1444,\n"," 'ea': 4646,\n"," 'photog': 12845,\n"," '##maid': 8283,\n"," '##isms': 16105,\n"," '1932': 9223,\n"," 'intelle': 4054,\n"," '##rifying': 5646,\n"," 'lawyer': 8334,\n"," 'anime': 2317,\n"," '##ene': 1778,\n"," 'caval': 13577,\n"," 'mighty': 6382,\n"," 'loony': 11900,\n"," 'stag': 7950,\n"," 'schwar': 12402,\n"," 'lincoln': 7768,\n"," 'edd': 15492,\n"," '##ifications': 16555,\n"," 'seaga': 16385,\n"," '##oying': 12040,\n"," 'interact': 10552,\n"," 'britis': 12541,\n"," 'swoon': 12534,\n"," 'manhattan': 6565,\n"," 'authent': 10786,\n"," 'charism': 8099,\n"," '##y': 108,\n"," 'cellul': 6052,\n"," 'outsiders': 16663,\n"," 'intentionally': 10909,\n"," 'brash': 7258,\n"," '##fee': 15125,\n"," '##yon': 8776,\n"," '##oof': 1548,\n"," 'notable': 5507,\n"," 'worse': 1429,\n"," 'wonderland': 9039,\n"," 'shrew': 9535,\n"," 'willie': 12190,\n"," 'kannathil': 11278,\n"," 'turtle': 16192,\n"," '##bitsch': 7186,\n"," 'philipines': 17171,\n"," 'dick': 2605,\n"," 'nowad': 6335,\n"," 'hark': 7810,\n"," 'blythe': 18053,\n"," '40': 2887,\n"," 'paulo': 12664,\n"," 'silverman': 7020,\n"," 'deserv': 12722,\n"," 'fitting': 8655,\n"," 'serend': 15115,\n"," 'enforcement': 18188,\n"," 'interp': 5037,\n"," 'audienc': 13131,\n"," 'oxford': 17858,\n"," 'serving': 12114,\n"," 'children': 1930,\n"," 'brook': 2743,\n"," 'entranced': 17783,\n"," 'bardem': 13405,\n"," 'cunningham': 17823,\n"," 'plough': 14935,\n"," 'colors': 7439,\n"," 'wandered': 18135,\n"," 'ewo': 13926,\n"," 'carlitos': 16413,\n"," 'jenni': 13078,\n"," 'legitimate': 16256,\n"," 'fire': 2417,\n"," 'cous': 5748,\n"," 'reiser': 10015,\n"," '00pm': 16911,\n"," 'locations': 6882,\n"," 'studen': 12493,\n"," 'gold': 1523,\n"," 'anders': 6702,\n"," 'convolut': 13144,\n"," 'boris': 4551,\n"," '##bers': 7862,\n"," '##ribe': 14771,\n"," 'depicted': 16978,\n"," 'gotten': 8093,\n"," 'recomm': 1480,\n"," '##sheba': 12422,\n"," '##www': 9402,\n"," 'nurses': 13649,\n"," 'suprem': 6095,\n"," '##amenco': 14664,\n"," 'chor': 6732,\n"," '##iation': 5801,\n"," 'natural': 9340,\n"," 'choke': 10837,\n"," '##ifiably': 16739,\n"," 'monday': 17722,\n"," 'woe': 8244,\n"," '##vest': 3676,\n"," 'gersh': 9782,\n"," 'considers': 15587,\n"," 'representation': 9580,\n"," 'thought': 480,\n"," '##ater': 3905,\n"," 'library': 4283,\n"," '##kien': 11653,\n"," 'beta': 12214,\n"," 'reput': 4581,\n"," 'look': 463,\n"," 'gather': 4890,\n"," 'hazzard': 7956,\n"," '##zel': 9931,\n"," 'frus': 14717,\n"," 'flies': 15221,\n"," 'cont': 1069,\n"," 'sg': 7169,\n"," 'ready': 4983,\n"," 'burns': 6087,\n"," '7th': 9737,\n"," 'retains': 18158,\n"," '##gar': 2567,\n"," 'satisf': 4763,\n"," 'releasing': 10469,\n"," 'line': 1313,\n"," 'disgrace': 5366,\n"," 'strained': 15981,\n"," 'phys': 4387,\n"," 'piper': 9454,\n"," 'weaknesses': 17671,\n"," '##acular': 15435,\n"," 'readi': 12144,\n"," '##ortun': 1153,\n"," 'butch': 6743,\n"," 'psychological': 4820,\n"," 'satanic': 12433,\n"," 'blue': 2863,\n"," 'goldsw': 7542,\n"," 'scr': 2929,\n"," 'fields': 13436,\n"," 'easi': 15771,\n"," 'dk': 13896,\n"," '##hal': 4926,\n"," 'linear': 10899,\n"," 'ta': 2834,\n"," '##etball': 5506,\n"," 'fata': 8353,\n"," 'species': 9509,\n"," 'atmo': 13140,\n"," '##rated': 1508,\n"," 'sk': 1594,\n"," 'ginger': 6599,\n"," 'backs': 15367,\n"," 'pang': 14079,\n"," 'blasting': 17632,\n"," 'thro': 4195,\n"," '##oose': 7257,\n"," 'quirky': 3868,\n"," 'istan': 6687,\n"," 'solution': 10705,\n"," 'noti': 11845,\n"," 'lousy': 3531,\n"," 'considere': 13552,\n"," 'intelligently': 16500,\n"," 'oss': 5771,\n"," 'thru': 6309,\n"," 'hours': 2032,\n"," 'doubts': 8194,\n"," ...}"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"lIID_rquXa37","executionInfo":{"status":"ok","timestamp":1625504078846,"user_tz":-180,"elapsed":284,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"c07f735c-017e-4914-f4f3-87d9294274de"},"source":["!mkdir tokenizer\n","bert_wordpiece_tokenizer.save_model(\"tokenizer\")"],"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['tokenizer/vocab.txt']"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"JF5_gklbXeeu","executionInfo":{"status":"ok","timestamp":1625504081264,"user_tz":-180,"elapsed":286,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["tokenizer = BertWordPieceTokenizer.from_file(\"tokenizer/vocab.txt\")"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"yaKwOs23boQh","executionInfo":{"status":"ok","timestamp":1625504082326,"user_tz":-180,"elapsed":3,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":[""],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"R-ANSWWKXz29","executionInfo":{"status":"ok","timestamp":1625504082616,"user_tz":-180,"elapsed":4,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["tokenized_sentence = tokenizer.encode(\"Oh it works just fine\")"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hC11u5e-X35m","executionInfo":{"status":"ok","timestamp":1625504084755,"user_tz":-180,"elapsed":285,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"d91dd82c-185c-4113-84b2-80d5bc6579ac"},"source":["tokenized_sentence.tokens"],"execution_count":12,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['[CLS]', 'oh', 'it', 'works', 'just', 'fine', '[SEP]']"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"0f5gF5m4X5m9","executionInfo":{"status":"ok","timestamp":1625504086430,"user_tz":-180,"elapsed":300,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["tokenized_sentence = tokenizer.encode(\"ohoh i thougt it might be workingg well\")"],"execution_count":13,"outputs":[]},{"cell_type":"code","metadata":{"id":"z7-Nt9n2YA8E","executionInfo":{"status":"ok","timestamp":1625504091912,"user_tz":-180,"elapsed":4267,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["from transformers import BertTokenizerFast \n","tokenizer = BertTokenizerFast.from_pretrained(\"tokenizer\") "],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xHwTfLvKYGKA","executionInfo":{"status":"ok","timestamp":1625504097948,"user_tz":-180,"elapsed":4931,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"dbcad4ab-e74f-4894-cb15-bed974a672f3"},"source":["from transformers import LineByLineTextDataset \n","dataset = LineByLineTextDataset(tokenizer=tokenizer, file_path=\"corpus.txt\", block_size=128) "],"execution_count":15,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/transformers/data/datasets/language_modeling.py:124: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_mlm.py\n","  FutureWarning,\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"njU0BD1UYMVE","executionInfo":{"status":"ok","timestamp":1625504101782,"user_tz":-180,"elapsed":283,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["from transformers import DataCollatorForLanguageModeling \n","data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=True, mlm_probability=0.15) "],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"id":"OBM3SvrHYTNH","executionInfo":{"status":"ok","timestamp":1625504103292,"user_tz":-180,"elapsed":396,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["from transformers import TrainingArguments \n","training_args = TrainingArguments(output_dir=\"BERT\", overwrite_output_dir=True, num_train_epochs=1, per_device_train_batch_size=128) "],"execution_count":17,"outputs":[]},{"cell_type":"code","metadata":{"id":"_u5_N_-9YYqw","executionInfo":{"status":"ok","timestamp":1625504107377,"user_tz":-180,"elapsed":2967,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["from transformers import BertConfig, BertForMaskedLM \n","bert = BertForMaskedLM(BertConfig()) "],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"id":"JQY0-ZLvYYtm","executionInfo":{"status":"ok","timestamp":1625504121843,"user_tz":-180,"elapsed":12611,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["from transformers import Trainer \n","trainer = Trainer(model=bert, args=training_args, data_collator=data_collator, train_dataset=dataset) "],"execution_count":19,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":334},"id":"bVZDxhXBYYym","executionInfo":{"status":"ok","timestamp":1625504407077,"user_tz":-180,"elapsed":284148,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"bfcb2ab4-7cbd-4e76-d6d7-673263d56d6a"},"source":["trainer.train()"],"execution_count":20,"outputs":[{"output_type":"stream","text":["***** Running training *****\n","  Num examples = 50022\n","  Num Epochs = 1\n","  Instantaneous batch size per device = 128\n","  Total train batch size (w. parallel, distributed & accumulation) = 128\n","  Gradient Accumulation steps = 1\n","  Total optimization steps = 391\n"],"name":"stderr"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='391' max='391' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [391/391 04:42, Epoch 1/1]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table><p>"],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","\n","Training completed. Do not forget to share your model on huggingface.co/models =)\n","\n","\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=391, training_loss=5.378278727421675, metrics={'train_runtime': 283.7543, 'train_samples_per_second': 176.286, 'train_steps_per_second': 1.378, 'total_flos': 812585139730200.0, 'train_loss': 5.378278727421675, 'epoch': 1.0})"]},"metadata":{"tags":[]},"execution_count":20}]},{"cell_type":"code","metadata":{"id":"6uzttHnJYY1C","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625504411937,"user_tz":-180,"elapsed":2154,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"50607221-359b-4601-f3b4-04004181a2a7"},"source":["trainer.save_model(\"MyBERT\")"],"execution_count":21,"outputs":[{"output_type":"stream","text":["Saving model checkpoint to MyBERT\n","Configuration saved in MyBERT/config.json\n","Model weights saved in MyBERT/pytorch_model.bin\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"M-j4LqJHYs0F","executionInfo":{"status":"ok","timestamp":1625504413939,"user_tz":-180,"elapsed":286,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"6b8098f5-338c-4b27-b06a-762272a5120b"},"source":["from transformers import BertConfig \n","BertConfig() "],"execution_count":22,"outputs":[{"output_type":"execute_result","data":{"text/plain":["BertConfig {\n","  \"attention_probs_dropout_prob\": 0.1,\n","  \"gradient_checkpointing\": false,\n","  \"hidden_act\": \"gelu\",\n","  \"hidden_dropout_prob\": 0.1,\n","  \"hidden_size\": 768,\n","  \"initializer_range\": 0.02,\n","  \"intermediate_size\": 3072,\n","  \"layer_norm_eps\": 1e-12,\n","  \"max_position_embeddings\": 512,\n","  \"model_type\": \"bert\",\n","  \"num_attention_heads\": 12,\n","  \"num_hidden_layers\": 12,\n","  \"pad_token_id\": 0,\n","  \"position_embedding_type\": \"absolute\",\n","  \"transformers_version\": \"4.8.2\",\n","  \"type_vocab_size\": 2,\n","  \"use_cache\": true,\n","  \"vocab_size\": 30522\n","}"]},"metadata":{"tags":[]},"execution_count":22}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MyY0_v-eYs2k","executionInfo":{"status":"ok","timestamp":1625504416744,"user_tz":-180,"elapsed":365,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"df0a4444-a558-4896-d0b0-39b97c103646"},"source":["tiny_bert_config = BertConfig(max_position_embeddings=512, hidden_size=128, num_attention_heads=2, num_hidden_layers=2, intermediate_size=512) \n","tiny_bert_config "],"execution_count":23,"outputs":[{"output_type":"execute_result","data":{"text/plain":["BertConfig {\n","  \"attention_probs_dropout_prob\": 0.1,\n","  \"gradient_checkpointing\": false,\n","  \"hidden_act\": \"gelu\",\n","  \"hidden_dropout_prob\": 0.1,\n","  \"hidden_size\": 128,\n","  \"initializer_range\": 0.02,\n","  \"intermediate_size\": 512,\n","  \"layer_norm_eps\": 1e-12,\n","  \"max_position_embeddings\": 512,\n","  \"model_type\": \"bert\",\n","  \"num_attention_heads\": 2,\n","  \"num_hidden_layers\": 2,\n","  \"pad_token_id\": 0,\n","  \"position_embedding_type\": \"absolute\",\n","  \"transformers_version\": \"4.8.2\",\n","  \"type_vocab_size\": 2,\n","  \"use_cache\": true,\n","  \"vocab_size\": 30522\n","}"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":334},"id":"ByC2zBtwYs44","executionInfo":{"status":"ok","timestamp":1625504442147,"user_tz":-180,"elapsed":21979,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"fede53a9-694d-4fde-f256-3c98780b9b9c"},"source":["tiny_bert = BertForMaskedLM(tiny_bert_config) \n","trainer = Trainer(model=tiny_bert, args=training_args, data_collator=data_collator, train_dataset=dataset) \n","trainer.train() "],"execution_count":24,"outputs":[{"output_type":"stream","text":["***** Running training *****\n","  Num examples = 50022\n","  Num Epochs = 1\n","  Instantaneous batch size per device = 128\n","  Total train batch size (w. parallel, distributed & accumulation) = 128\n","  Gradient Accumulation steps = 1\n","  Total optimization steps = 391\n"],"name":"stderr"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='391' max='391' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [391/391 00:21, Epoch 1/1]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table><p>"],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","\n","Training completed. Do not forget to share your model on huggingface.co/models =)\n","\n","\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=391, training_loss=8.895718235493925, metrics={'train_runtime': 21.5776, 'train_samples_per_second': 2318.234, 'train_steps_per_second': 18.121, 'total_flos': 32771457490200.0, 'train_loss': 8.895718235493925, 'epoch': 1.0})"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["5661f9afceac41cd81911ec61f840d1c","0fe97a5a63614b35b1a58cb8d9765242","6295bed1bbfc43ff8bd374152775b094","91e3626b49814c7e85fc351d6cea403c","cd7b13f0ba0f44a88b1d085fb5acf789","24e111b1fc5746a8a69f862a0dc3bc7d","e345fb951df24e0d848a051a485b4b65","86b7927b6dc44fb28cf3b006163c39cf","133035cbbade4827bc6764218387a54b","e4efd2d54399463db446591d18cbe6fd","c1e03b6d025a4dba9e37eb35c48d93ec","62399066111345ec827e1912ec1d97ba","73ca0820cf8a482bb3cb4bd658e7a5d3","2989c05594ea4054b311517ce5339ed2","ef71247bb53a4c709d59e63ab7d60c33","62b92d97fdcb49b1ade6d076d0b203eb","94a9c2fcf79f4ac0a8f2d712411f7d2b","d8e8c28122024aadb70ff1aab5530eeb","7a9083129f274149a176a0e31eb48411","dac236c954ef4b0f99067b8bbeba6283","96ad3e5ee6bc4398a0c6e8ea948682a4","3f2e7ff1d0ad466196075aad10b8b84a","9fb9172e15b3405fab9a7db14d0c560d","48f55d9d4e3046d5abe21fef5b2a377c","7d102900f8884e44aa4953d89ad79465","6946d2105b2e4fd1bfab2b10f1256053","0b351be38ce9450eb4f4a857132b0a79","c41a59274c78422fa1c846a1711b2439","477900896cec431f98ed705f5fbbf78b","6cead6beda334e7a8db3f27ad5845705","d26b4d4dbcd243cd8933f81fffad0b4d","a0d2c4f1560a4a6bab0be1285c932283","9dac89e1bf484a1fafaa1b5235846d93","2290a421c99944a19da4fa1888204a0a","3fe4f71880a5497e950d15a381614651","7fc1dee184a54de987dbba3af55c597e","4e9a142be38247088fe4f3a7d6ecd6f3","127e05f39b2d4b0f84dd7ca57d71cc98","73de1777b2f5478891fc9b8dc46c26a0","476bdde5624144919c7abb27938c7221"]},"id":"fMCmbDfFYs_s","executionInfo":{"status":"ok","timestamp":1625504468368,"user_tz":-180,"elapsed":22470,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"b8f7f093-cc44-465f-d412-5a38025c9f20"},"source":["from transformers import TFBertModel, BertTokenizerFast \n","bert = TFBertModel.from_pretrained(\"bert-base-uncased\") \n","tokenizer = BertTokenizerFast.from_pretrained(\"bert-base-uncased\") \n","bert.layers "],"execution_count":25,"outputs":[{"output_type":"stream","text":["https://huggingface.co/bert-base-uncased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp_jskfvy4\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"5661f9afceac41cd81911ec61f840d1c","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["storing https://huggingface.co/bert-base-uncased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n","creating metadata file for /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n","loading configuration file https://huggingface.co/bert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n","Model config BertConfig {\n","  \"architectures\": [\n","    \"BertForMaskedLM\"\n","  ],\n","  \"attention_probs_dropout_prob\": 0.1,\n","  \"gradient_checkpointing\": false,\n","  \"hidden_act\": \"gelu\",\n","  \"hidden_dropout_prob\": 0.1,\n","  \"hidden_size\": 768,\n","  \"initializer_range\": 0.02,\n","  \"intermediate_size\": 3072,\n","  \"layer_norm_eps\": 1e-12,\n","  \"max_position_embeddings\": 512,\n","  \"model_type\": \"bert\",\n","  \"num_attention_heads\": 12,\n","  \"num_hidden_layers\": 12,\n","  \"pad_token_id\": 0,\n","  \"position_embedding_type\": \"absolute\",\n","  \"transformers_version\": \"4.8.2\",\n","  \"type_vocab_size\": 2,\n","  \"use_cache\": true,\n","  \"vocab_size\": 30522\n","}\n","\n","https://huggingface.co/bert-base-uncased/resolve/main/tf_model.h5 not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp574a63hw\n"],"name":"stderr"},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"133035cbbade4827bc6764218387a54b","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=536063208.0, style=ProgressStyle(descri…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["storing https://huggingface.co/bert-base-uncased/resolve/main/tf_model.h5 in cache at /root/.cache/huggingface/transformers/775efbdc2152093295bc5824dee96da82a5f3c1f218dfface1b8cef3094bdf8f.c719a806caef7d36ec0185f14b3b5fa727d919f924abe35622b4b7147bfbb8c7.h5\n","creating metadata file for /root/.cache/huggingface/transformers/775efbdc2152093295bc5824dee96da82a5f3c1f218dfface1b8cef3094bdf8f.c719a806caef7d36ec0185f14b3b5fa727d919f924abe35622b4b7147bfbb8c7.h5\n","loading weights file https://huggingface.co/bert-base-uncased/resolve/main/tf_model.h5 from cache at /root/.cache/huggingface/transformers/775efbdc2152093295bc5824dee96da82a5f3c1f218dfface1b8cef3094bdf8f.c719a806caef7d36ec0185f14b3b5fa727d919f924abe35622b4b7147bfbb8c7.h5\n"],"name":"stderr"},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"stream","text":["Some layers from the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']\n","- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","All the layers of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n","If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n","https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmptjrcqhtk\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"94a9c2fcf79f4ac0a8f2d712411f7d2b","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["storing https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt in cache at /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n","creating metadata file for /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n","https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp83epubcm\n"],"name":"stderr"},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"7d102900f8884e44aa4953d89ad79465","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["storing https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n","creating metadata file for /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n"],"name":"stderr"},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"stream","text":["https://huggingface.co/bert-base-uncased/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpgk2_ycuj\n"],"name":"stderr"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9dac89e1bf484a1fafaa1b5235846d93","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["storing https://huggingface.co/bert-base-uncased/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/c1d7f0a763fb63861cc08553866f1fc3e5a6f4f07621be277452d26d71303b7e.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n","creating metadata file for /root/.cache/huggingface/transformers/c1d7f0a763fb63861cc08553866f1fc3e5a6f4f07621be277452d26d71303b7e.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n","loading file https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n","loading file https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n","loading file https://huggingface.co/bert-base-uncased/resolve/main/added_tokens.json from cache at None\n","loading file https://huggingface.co/bert-base-uncased/resolve/main/special_tokens_map.json from cache at None\n","loading file https://huggingface.co/bert-base-uncased/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/c1d7f0a763fb63861cc08553866f1fc3e5a6f4f07621be277452d26d71303b7e.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n"],"name":"stderr"},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["[<transformers.models.bert.modeling_tf_bert.TFBertMainLayer at 0x7f5b9fd316d0>]"]},"metadata":{"tags":[]},"execution_count":25}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Z-tQO7EbYtCR","executionInfo":{"status":"ok","timestamp":1625504473380,"user_tz":-180,"elapsed":325,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"5390c3db-44ea-437f-e812-354840dcbfa1"},"source":["tokenized_text = tokenizer.batch_encode_plus([\"hello how is it going with you\",\"lets test it\"], return_tensors=\"tf\", max_length=256, truncation=True, pad_to_max_length=True) \n","bert(tokenized_text) "],"execution_count":26,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2132: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n","  FutureWarning,\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["TFBaseModelOutputWithPooling([('last_hidden_state',\n","                               <tf.Tensor: shape=(2, 256, 768), dtype=float32, numpy=\n","                               array([[[ 1.00471288e-01,  6.77022934e-02, -8.33591744e-02, ...,\n","                                        -4.93304461e-01,  1.16539642e-01,  2.26646975e-01],\n","                                       [ 3.23624432e-01,  3.70718002e-01,  6.14686370e-01, ...,\n","                                        -6.27267480e-01,  3.79082561e-01,  7.05312043e-02],\n","                                       [ 1.99534193e-01, -8.75509918e-01, -6.47860616e-02, ...,\n","                                        -1.28080100e-02,  3.07651967e-01, -2.07310896e-02],\n","                                       ...,\n","                                       [-6.53299540e-02,  1.19045913e-01,  5.76846719e-01, ...,\n","                                        -2.95459926e-01,  2.49742977e-02,  1.13964222e-01],\n","                                       [-2.64715403e-01, -7.86383227e-02,  5.47281384e-01, ...,\n","                                        -1.37515306e-01, -5.94685934e-02, -5.17925322e-02],\n","                                       [-2.44959027e-01, -1.14799343e-01,  5.92174053e-01, ...,\n","                                        -1.56881928e-01, -3.39758471e-02, -8.46135020e-02]],\n","                               \n","                                      [[ 2.94565968e-02,  2.30868325e-01,  2.92651832e-01, ...,\n","                                        -1.30421251e-01,  1.89659148e-01,  4.68428344e-01],\n","                                       [ 1.70523262e+00,  6.91359818e-01,  7.31509924e-01, ...,\n","                                         2.89304137e-01,  5.36758423e-01, -1.54552370e-01],\n","                                       [ 1.04597911e-01,  9.63677615e-02,  6.99652955e-02, ...,\n","                                        -4.15922850e-01, -1.18989676e-01, -6.72240913e-01],\n","                                       ...,\n","                                       [ 8.00909936e-01,  2.38983527e-01,  4.15492773e-01, ...,\n","                                         3.90535370e-02,  2.34373271e-01,  1.22278899e-01],\n","                                       [ 2.60863423e-01,  4.43267561e-02,  3.63649219e-01, ...,\n","                                        -7.53857195e-04,  3.84623855e-02, -2.14213550e-01],\n","                                       [-2.30111465e-01, -4.98387933e-01, -1.26490444e-02, ...,\n","                                         4.49868321e-01,  6.16021976e-02, -2.61357427e-01]]],\n","                                     dtype=float32)>),\n","                              ('pooler_output',\n","                               <tf.Tensor: shape=(2, 768), dtype=float32, numpy=\n","                               array([[-0.9204854 , -0.37138984, -0.6051261 , ..., -0.4473696 ,\n","                                       -0.6434759 ,  0.9423271 ],\n","                                      [-0.88541585, -0.26547667,  0.21014938, ...,  0.17237104,\n","                                       -0.640299  ,  0.88883436]], dtype=float32)>)])"]},"metadata":{"tags":[]},"execution_count":26}]},{"cell_type":"code","metadata":{"id":"DMmRg2X-Y7LF","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625504480743,"user_tz":-180,"elapsed":4230,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"6a4edc0c-0d1a-4033-c44a-5476e5ea0914"},"source":["from tensorflow import keras \n","import tensorflow as tf \n","max_length = 256 \n","tokens = keras.layers.Input(shape=(max_length,), dtype=tf.dtypes.int32) \n","masks = keras.layers.Input(shape=(max_length,), dtype=tf.dtypes.int32) \n","embedding_layer = bert.layers[0]([tokens,masks])[0][:,0,:] \n","dense = tf.keras.layers.Dense(units=2, activation=\"softmax\")(embedding_layer) \n","model = keras.Model([tokens,masks],dense) "],"execution_count":27,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/array_ops.py:5049: calling gather (from tensorflow.python.ops.array_ops) with validate_indices is deprecated and will be removed in a future version.\n","Instructions for updating:\n","The `validate_indices` argument has no effect. Indices are always validated on CPU and never validated on GPU.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"T8xeS5TyY7No","executionInfo":{"status":"ok","timestamp":1625504483580,"user_tz":-180,"elapsed":312,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"34427854-5ba6-41e2-ec99-88b82ec59505"},"source":["tokenized = tokenizer.batch_encode_plus([\"hello how is it going with you\",\"hello how is it going with you\"], return_tensors=\"tf\", max_length= max_length, truncation=True, pad_to_max_length=True) "],"execution_count":28,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2132: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n","  FutureWarning,\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AGMwpG8OY7P_","executionInfo":{"status":"ok","timestamp":1625504485127,"user_tz":-180,"elapsed":299,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"fd5fdad8-8545-418e-9a4b-378c3a79e934"},"source":["model([tokenized[\"input_ids\"],tokenized[\"attention_mask\"]]) "],"execution_count":29,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n","array([[0.56051165, 0.43948835],\n","       [0.56051165, 0.43948835]], dtype=float32)>"]},"metadata":{"tags":[]},"execution_count":29}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"b24ahO1UY7SR","executionInfo":{"status":"ok","timestamp":1625504487398,"user_tz":-180,"elapsed":339,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"41ea32ad-1ad7-4a2b-d525-446ca88f4cbb"},"source":["model.compile(optimizer=\"Adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]) \n","model.summary() "],"execution_count":30,"outputs":[{"output_type":"stream","text":["Model: \"model\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","input_1 (InputLayer)            [(None, 256)]        0                                            \n","__________________________________________________________________________________________________\n","input_2 (InputLayer)            [(None, 256)]        0                                            \n","__________________________________________________________________________________________________\n","bert (TFBertMainLayer)          TFBaseModelOutputWit 109482240   input_1[0][0]                    \n","                                                                 input_2[0][0]                    \n","__________________________________________________________________________________________________\n","tf.__operators__.getitem (Slici (None, 768)          0           bert[0][0]                       \n","__________________________________________________________________________________________________\n","dense (Dense)                   (None, 2)            1538        tf.__operators__.getitem[0][0]   \n","==================================================================================================\n","Total params: 109,483,778\n","Trainable params: 109,483,778\n","Non-trainable params: 0\n","__________________________________________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XSCi31iUZL1Y","executionInfo":{"status":"ok","timestamp":1625504491806,"user_tz":-180,"elapsed":291,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}}},"source":["model.layers[2].trainable = False "],"execution_count":31,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wcfBTBw8ZL6V","executionInfo":{"status":"ok","timestamp":1625504525810,"user_tz":-180,"elapsed":30123,"user":{"displayName":"meysam asgari chenaghlu","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgN-uRJ5KciT9X_sZpZ4pOpEANlj5GCtXuY2ecq=s64","userId":"13491140841572947804"}},"outputId":"08436178-6e86-4151-f483-2c2671ef7cb0"},"source":["import pandas as pd \n","imdb_df = pd.read_csv(\"IMDB Dataset.csv\") \n","reviews = list(imdb_df.review) \n","tokenized_reviews = tokenizer.batch_encode_plus(reviews, return_tensors=\"tf\", max_length=max_length, truncation=True, pad_to_max_length=True) \n","\n","import numpy as np \n","train_split = int(0.8 * len(tokenized_reviews[\"attention_mask\"])) \n","train_tokens = tokenized_reviews[\"input_ids\"][:train_split] \n","test_tokens = tokenized_reviews[\"input_ids\"][train_split:] \n","train_masks = tokenized_reviews[\"attention_mask\"][:train_split] \n","test_masks = tokenized_reviews[\"attention_mask\"][train_split:] \n","sentiments = list(imdb_df.sentiment) \n","labels = np.array([[0,1] if sentiment == \"positive\" else [1,0] for sentiment in sentiments]) \n","train_labels = labels[:train_split] \n","test_labels = labels[train_split:] "],"execution_count":32,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2132: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n","  FutureWarning,\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"YaQlIkdZZL8c","colab":{"base_uri":"https://localhost:8080/"},"outputId":"faaa116b-6cc2-4263-876c-bebf162eafe6"},"source":["model.fit([train_tokens,train_masks],train_labels, epochs=5)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Epoch 1/5\n","WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3RTyDkLzZL-8"},"source":[""],"execution_count":null,"outputs":[]}]}